import requests
import parsel
import json
import csv


def get_html(url, params, headers, cookies):
    try:
        resp = requests.get(url=url, headers=headers, params=params, cookies=cookies)
        resp.encoding = resp.apparent_encoding
        return resp.text
    except BaseException as e:
        print(e)
        return None

def parse_html(html):
    datas = []
    selector = parsel.Selector(text=html)
    json_str = selector.css('#__NEXT_DATA__::text').get()
    json_dict = json.loads(json_str)
    result = json_dict['props']['pageProps']['initData']['content']['positionResult']['result']

    for rlt in result:
        dic = {
            'company_full_name': rlt['companyFullName'],
            'type': rlt['firstType'] + '/' + rlt['secondType'] + '/' + rlt['thirdType'],
            'city_district': rlt['city'] + '/' + rlt['district'],
            'salary': rlt['salary'],
            'education': rlt['education'],
            'work_year': rlt['workYear']
        }
        datas.append(dic)

    return datas


def save_data(datas):
    try:
        header = ['company_full_name', 'type', 'city_district', 'salary', 'education', 'work_year']
        with open('./职位数据.csv', 'w', encoding='utf-8-sig', newline='') as f:
            csv_writer = csv.DictWriter(f, fieldnames=header)
            csv_writer.writeheader()
            csv_writer.writerows(datas)
        return 1
    except:
        return None


def main():
    url = 'https://www.lagou.com/wn/jobs'
    params = {
        'fromSearch': 'true', 
        'suginput': 'python',
        'kd': 'python', 
        'city': '长沙',
        'pn': '1',
    }
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.35',
    }
    cookies = {
        "JSESSIONID": "ABAAAECABIEACCA7B86C32CA17A32AFBC5E939092D77C68",
        "WEBTJ-ID": "05132023%2C104847-1881302c177549-09d8dbd51e8e64-7b515477-1327104-1881302c178ad1",
        "RECOMMEND_TIP": "true",
        "privacyPolicyPopup": "false",
        "user_trace_token": "20230513104850-6fd0283d-6f68-422d-b17b-f503a41cd2d6",
        "LGSID": "20230513104850-666930de-35bd-4743-8963-d18470bf322b",
        "PRE_UTM": "",
        "PRE_HOST": "",
        "PRE_SITE": "https%3A%2F%2Fwww.lagou.com",
        "PRE_LAND": "https%3A%2F%2Fwww.lagou.com%2F",
        "LGUID": "20230513104850-4d6cdc77-8a51-4ce4-93d4-85c53f20c0a7",
        "_ga": "GA1.2.1339973166.1683946131",
        "_gid": "GA1.2.1345082021.1683946131",
        "Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6": "1683946131",
        "sajssdk_2015_cross_new_user": "1",
        "sensorsdata2015session": "%7B%7D",
        "X_MIDDLE_TOKEN": "dcfa6ce9f2511006245291d3457d290e",
        "__lg_stoken__": "2a1bcea0ea1991f5b2fab4c1486ae82c9868fb0995b8302ee498175a8a511618d04065745fbb9cd7041e5724e8addaebde77f1317e711ad21238ab828da338795812d0fa11a7",
        "gate_login_token": "v1####c8d8dc9ae3f7464707d69d50765455e17bada3274d4c62af241abb340cd20f5d",
        "LG_LOGIN_USER_ID": "v1####bc0b931384714ddef9d19099ffcb81ae5433684f5d2dedfaa96927b63f1d0526",
        "LG_HAS_LOGIN": "1",
        "_putrc": "3F954D2A4D71C509123F89F2B170EADC",
        "login": "true",
        "unick": "%E7%8E%8B%E6%AC%A3",
        "showExpriedIndex": "1",
        "showExpriedCompanyHome": "1",
        "showExpriedMyPublish": "1",
        "hasDeliver": "25",
        "index_location_city": "%E5%85%A8%E5%9B%BD",
        "X_HTTP_TOKEN": "ee16fac7bf9dcd4f7466493861db16e8e9474434e8",
        "Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6": "1683946647",
        "LGRID": "20230513105732-0f460d6f-caef-4826-a82e-f13d4d7a1a82",
        "__SAFETY_CLOSE_TIME__13056399": "1",
        "sensorsdata2015jssdkcross": "%7B%22distinct_id%22%3A%2213056399%22%2C%22first_id%22%3A%221881302d15b662-074800df376ade-7b515477-1327104-1881302d15c4dd%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24os%22%3A%22Windows%22%2C%22%24browser%22%3A%22Chrome%22%2C%22%24browser_version%22%3A%22113.0.0.0%22%7D%2C%22%24device_id%22%3A%221881302d15b662-074800df376ade-7b515477-1327104-1881302d15c4dd%22%7D"
    }
    html = get_html(url, params, headers, cookies)
    if html:
        print('获取网页源代码成功！')
        datas = parse_html(html)
        if datas:
            print('解析职位数据成功！')
            if save_data(datas):
                print('保存职位数据成功！')

if __name__ == '__main__':
    main()

